#!/bin/bash

# mta-track.sh
# 
# Script to query in parallel mail servers and find out what MTA (mail
# transfer agent, a.k.a. mail exchange system) they are using.
#
# This script requires the script get-mta-banner.sh to run.
#
# Syntax
#
#    ./mta-track.sh [-h] [-n N] log-server.txt mta.txt
#
# where log-server.txt is a text file obtained with server-track.sh,
# and mta.txt is a text file with a list of lines with the format
#
# STRING,TAG,TYPE,URL
#
# where STRING is the expected string for a given MTA in the telnet
# session (e.g. "Microsoft Exchange"), TAG is a tag that will be used
# to identify each MTA, TYPE is either "mta" or "filter" and URL is
# the web address of the project/company's homepage.
#
# Results are sent to stdout.
#
# Each line of the output has one of the following formats ("string"
# means that the string is literal, but the "" don't appear in the
# actual output):
#
# mail server \t "server not found"
# mail server \t "unknown MTA"
# mail server \t TAG \t telnet line
#
# v0.8  24 Aug 2007
#
# Author: Ramon Casero <ramon.casero@oucs.ox.ac.uk>
# Copyright © 2007 University of Oxford
#
# This file is part of mta-tracker.
#
# mta-tracker is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# mta-tracker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with mta-tracker.  If not, see <http://www.gnu.org/licenses/>.
#
# ChangeLog
#
# * 2008-08-21 (rcasero):
#
#   ** Upgrade licence to GPL v3.
#
# * 2007-08-24 (rcasero): v0.8.
#
#   ** Script mta-track.sh has been split into 2 scripts:
#
#     - server-track.sh to generate a list of institutions, their domains
#       and corresponding mail servers
#
#     - mta-track.sh to query the mail servers and identify what MTAs
#       they are using
#
#   ** Take out all the functionality that is in server-track.sh.
#
# * 2007-08-24 (rcasero): v0.7.
#
#   ** Big rewrite. The way MTAs were identified from the telnet
#   output was as ugly as it can be. All that has been replaced by a
#   grep line. 
#
#   ** Produce 2 files for strings and tags of known MTAs.
#
# * 2007-07-16 (rcasero): v0.6.
#
#   ** Fix bug, MTA tags were not correctly found in the telnet
#   session, due to an incorrect an unnecessary find_mta_name()
#   function
#
#   ** Fix bug, get-institution-from-domain.sh was lacking the
#   relative path, and now it reads input from stdin
#
# * 2007-06-27 (rcasero): v0.5.
#
#   ** Script now waits for all subshells to finish before exiting
#
#   ** Rename get_mta_banner.sh as get-mta-banner.sh.
#
#   ** Take out code of registered_for() to new script.
#
# * 2007-06-19 (rcasero): v0.4.
#
#   ** Fix bug, "server not found" was being sent to stdout instead of
#      stderr, and this added unwanted lines to the log file
#
#   ** Add different formats of output lines to help header
#
# * 2007-06-18 (rcasero): v0.3.
#
#   ** Tidy up echo lines
#
#   ** Obtain "registered for:" field from whois for each domain. This
#      for the moment seems the only way to identify domain redundancy
#
#   ** Ignore first lines in telnet session. oxmail.ox.ac.uk was being
#      interpreted as having xMail
#
#   ** When INT is trapped, the script should exit, not kill itself
#      (thanks, Stuart)
#
# * 2007-06-14 (rcasero): v0.2.
#
#   ** Now several domains are queried in parallel, instead of going
#      one by one; given network latency and slowness in telnet
#      sessions, this should speed things up quite a lot
#
#   ** Trap CTRL-C signal so that all subshells are killed when script
# is interrupted
#
#   ** Use getopts to parse input arguments; add -h -n options
#
#   ** Redirect all output that is not MTA info to stderr to avoid
#      that it muddles up the log file, in case output is sent to a
#      file
#
#   ** Add function "usage" for syntax text
#
#   ** Make script header more verbose, and add comments where missing
#
#   ** Ignore comment lines in input files
#
# * 2007-06-12 (rcasero): v0.1. First working copy. 
#
#   ** Given a file with a list of domains, and a file with a list of
#      MTAs, it gets the mail servers from each domain, and checks
#      whether they are running a known MTA
#
#   ** Error checks performed: domain exists, domain has mail servers,
#      mail server exists, known MTA, unknown MTA, security through
#      obscurity
#
# * 2007-05-29 (rcasero): Start script.

# syntax message
usage() {
    echo "Syntax: `basename $0` [-h] [-n N] log-server.txt mta.txt"
    echo
    echo "  -n N"
    echo "    query N domains in parallel (default 20)"
    echo
    echo "  -h"
    echo "    this help"
}

# constants
FILEMTA_TAGS=mta-names-tags.txt
FILEMTA_STRS=mta-names-strs.txt

# defaults
PARALLEL=20 # maximum number of parallel queries

# kill all subshells forked from this script when it is interrupted
# (ctrl+c)
trap "jobs -p | xargs kill; exit 1" INT 

# parse option arguments 
while getopts  "hn:" option
do
  # parse options
  case "$option" in
      h) # help
	  usage
	  exit 0
	  ;;
      n) # number of parallel queries
	  PARALLEL=$OPTARG
	  ;;
  esac

done

# get ready to process non-option arguments (i.e. file names)
shift $((OPTIND-1)); OPTIND=1

# check number of input arguments
if [ ! $# -eq 2 ]
    then
    usage
    exit 1
fi

# parse non-option arguments
FILESRV=$1
FILEMTA=$2

# check that files with domain names and MTA info exist
if [ -e $FILESRV ]
    then
    echo Server names file: "$FILESRV" 1>&2
else
    echo Error! File cannot be found: "$FILESRV" 1>&2
    exit 1
fi

if [ -e $FILEMTA ]
    then
    echo MTA strings/tags file: "$FILEMTA" 1>&2
else
    echo Error! File cannot be found: "$FILEMTA" 1>&2
    exit 1
fi

# create a file of only MTA tags and another of only MTA id strings
egrep -v '^#' "$FILEMTA" | cut -d ',' -f 2 > "$FILEMTA_TAGS"
egrep -v '^#' "$FILEMTA" | cut -d ',' -f 1 > "$FILEMTA_STRS"

# check that there are the same number of tags and strings
if [ ! `cat "$FILEMTA_TAGS" | wc -l` -eq `cat "$FILEMTA_STRS" | wc -l` ];
    then
    echo Error! Number of MTA strings and tags don\'t match
    exit 1
fi

# loop through domains (ignoring comment lines)
while read server; 
  do
  
  # keep a maximum of PARALLEL queries running in parallel
  while [ `jobs -p | wc -l` -ge $PARALLEL ]; do
      sleep $(( RANDOM % 5 + 1 ))
  done
  
  # fork and send subshell to background
  (
      
  # server being queried
  echo Server: $server... 1>&2
      
  # check that the mailserver exists (has an IP address)
  # this is because I have found cases where a server is given in
  # the MX record of a server, but then the server has no entry in
  # the DNS
  case `host $server` in
      Host\ *\ not\ found*)
  echo server not found 1>&2
  echo -e "$server\t"server not found
  continue
  ;;
  esac

  # telnet to mail server to see welcome banner; we also send a
  # "help" command because sometimes the MTA system (e.g. qmail)
  # does not appear in the welcome banner, but it does in the help
  # response
  #
  # then, search for any known MTA string within the telnet output
  #
  # (ah, the sheer beauty of the linux shell and grep)
  telnetline=$(grep -m 1 -f "$FILEMTA_STRS" \
      <(./get-mta-banner.sh $server))

  # if an MTA has been identified
  if [[ $telnetline != "" ]];
      then
      # find what string was indentified
      string=$(grep -of "$FILEMTA_STRS" <(echo $telnetline))
      # get what tag that string corresponds to
      tag=$(grep "$string" "$FILEMTA" | cut -d ',' -f 2 )

      # output to stdout
      echo -e "$server\t$tag\t$telnetline"

  else

      # output to stdout
      echo -e "$server\t"unknown MTA
      
  fi
  
  exit 0
    
  ) & # end fork

# ignore comment lines, "domain not found" or "no MX record"
# forget about everything but server names
# sort server names
# remove duplicates
done < <(cat "$FILESRV" \
    | egrep -v -e '^#' -e "domain not found" -e "no MX record" \
    | cut -f 3 \
    | sort \
    | uniq ) # while read log-server line

# wait until all subshells have finished
while [ `jobs -p | wc -l` -gt 1 ]; do
    sleep $(( RANDOM % 5 + 1 ))
done

exit 0
